package irie.util;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.log4j.BasicConfigurator;

import irie.search.HTMLentity;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class PageCrawler implements PageProcessor {
	static ArrayList<HTMLentity> list = new ArrayList<HTMLentity>();
	int k = 1;
	private Site site = Site.me().setRetryTimes(3).setSleepTime(0);

	@Override
	public void process(Page page) {
		final String reg = "[^\u4e00-\u9fa5]";
		// page.addTargetRequests(page.getHtml().links().all());
		page.addTargetRequests(page.getHtml().links().regex("(http://mm.10086.cn/android/info/.*)").all());
		// page.putField("URL", page.getUrl().toString());
		String str = page.getHtml().toString();
		str = str.replaceAll(reg, "");
		page.putField("HTML", str);
		// System.out.println("=============================================================================");
		// System.out.println("=============================================================================");
		// MySegm ms = new MySegm();
		// ms.seg(str);
		HTMLentity hen = new HTMLentity();
		hen.setHid(k++);
		hen.setURL(page.getUrl().toString());
		hen.setContent(str);
		hen.setHtmlcontent(page.getHtml().toString());
		hen.setFeature(null);
		// System.out.println("-----------------------------"+hen);
		list.add(hen);
		if (k == 101) {
			try {
				this.output();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	public void output() throws IOException {
		File file = new File("D:\\webmagic\\entity", "html.txt");
	
			FileWriter fw = new FileWriter(file, true);
			Iterator it = list.iterator();
			while (it.hasNext()) {
				HTMLentity he = (HTMLentity) it.next();
				//File htmlfile = new File("D:\\webmagic\\mm.10086.cn", he.getHid() + ".txt");
				fw.write(String.valueOf(he.getHid()));
				fw.write(he.getURL());
				fw.write(he.getContent() + '\n');

//				FileWriter fw1 = new FileWriter(htmlfile, true);
//				fw1.write(he.getHtmlcontent());
//				fw1.close();
			}
			fw.close();
		
		Iterator itt = list.iterator();
		while(itt.hasNext()){
			HTMLentity he = (HTMLentity) itt.next();
			File htmlfile = new File("D:\\webmagic\\mm.10086.cn", he.getHid() + ".txt");
			FileWriter fw1 = new FileWriter(htmlfile);
			fw1.write(he.getHtmlcontent());
			fw1.flush();
			fw1.close();
		}
	}

	public static void main(String[] args) {
		BasicConfigurator.configure();
		PageCrawler pg = new PageCrawler();
		Spider.create(pg).addUrl("http://mm.10086.cn/").thread(1).run();
		// .addPipeline(new FilePipeline("D:\\webmagic\\"))
		// pg.output();
	}
}